# coding=utf-8

from queue import Queue
import threading
from time import sleep
import os
import json
import re
    
# each_down
import requests
from bs4 import BeautifulSoup 

        
class Each:

    def __init__(self, url, path):
        self.url = url
        self.path = path


class Statistics:
    
    def __init__(self):
        self.album = 0
        self.task = 0
        self.done = 0
        self.lock = threading.Lock()
        
    def __str__(self):
        return "album=%s task=%s done=%s" % (self.album, self.task, self.done)

    def add_album(self):
#         self.lock.acquire()
        self.album += 1
#         self.lock.release()
        
    def add_task(self):
#         self.lock.acquire()
        self.task += 1
#         self.lock.release()
        
    def add_done(self):
#         self.lock.acquire()
        self.done += 1
#         self.lock.release()


statistics = Statistics()

flag_run = True


def p(str1):
    if 1:
        print("%s" % str1)


header1 = {
    "X-Requested-With":"XMLHttpRequest",
    "User-Agent":"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.23 Safari/537.36",
    "Content-Type":"application/x-www-form-urlencoded",
    "Accept":"*/*",
    "Referer":"http://www.renren.com/SysHome.do",
    "Accept-Encoding":"gzip, deflate",
    "Accept-Language":"zh-CN,zh;q=0.9",
}

    
def login():
    
    session = requests.Session()
    # 抓包获取的form信息
    data1 = "（已删除）" 
    
    r = session.post("http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=2018762241620", headers=header1, data=data1)
    p(r.text)

    json1 = json.loads(r.text)
    try:
        return (json1['code'] == True), session
    except Exception as e:
        print(e)
        return False    


def list_2_each(session, q_each):
    global statistics
    
    r = session.post("http://friend.renren.com/groupsdata", headers=header1)
#     p(r.text)

    pat_friends = '"friends": \[.*\],'
    list1 = re.compile(pat_friends).findall(r.text)
    print(len(list1))
    pat_fid = '"fid":\d+'
    list2 = re.compile(pat_fid).findall(list1[0])
    print(len(list2))
    
    pat_hostFriendCount = '"hostFriendCount": [0-9]+,'
    list3 = re.compile(pat_hostFriendCount).findall(r.text)
    print(len(list3))
    pat_count = '\d+'
    list4 = re.compile(pat_count).findall(list3[0])
    print(list4[0])

    # 间接解析出来的好友数量，要等于直接解析出来的好友数量
    if len(list2) != int(list4[0]):
        return False
    
    for fid_obj in list2:
        fid = re.compile(pat_count).findall(fid_obj)
        if len(fid) == 1:
            a_link = "http://www.renren.com/profile.do?id=%s" % fid[0]
#             p("PUT %s" % a_link)
            q_each.put(a_link)
            statistics.add_task()
        else:
            print("error: len(fid)!=1 " + fid_obj)
            continue
        
    print("info: list_2_each OVER")


def each_down(session, q_each):

    global statistics
    
    while flag_run:
        try:
            each1 = q_each.get(timeout=30)
        except Exception as e:
            print(e)
            print("info: each_down 30s break")
            break
        
        p("GET:%s" % each1)
        for i in range(0, 3):
            try:
                response = session.get(each1, headers=header1)
#                 sleep(5)

                content = response.content.decode("utf-8", "ignore")
                soup = BeautifulSoup(content, "html.parser")
                visit_obj = soup.select('div.cover-bg h1')
                
                if visit_obj:
                    p(visit_obj)
                    statistics.add_done()
                else:
                    print(content)
                    os.system("pause")
                
#                 sleep(5)
                p("DONE %s" % each1)
                break
            except Exception as e:
                if i < 2:
                    sleep(1)
                else:
                    print(e)
                    
    print("info: each_down OVER")

    
if __name__ == '__main__':
    
    # 登陆
    ret, session = login()
    if not ret:
        print("error: login")
        os.system("pause")
        exit()
     
    threads = []
    q_each = Queue(maxsize=50)

    # 2、有效页->每张单独页
    for i in range(0, 1):
        t = threading.Thread(target=list_2_each, args=(session, q_each))
        threads.append(t)
     
    # 3、每张单独页下载
    for i in range(0, 10):
        t = threading.Thread(target=each_down, args=(session, q_each))
        threads.append(t)
 
    # 开干
    count = range(len(threads))
    for i in count:
        threads[i].start()
 
    i = 0
    while True:
        sleep(5)
        print("info: q_each=%s %s" % (q_each.qsize(), statistics))
        if q_each.qsize() == 0:
            if i < 10:
                i += 1
            else:
                break
          
    # 收工
    flag_run = False
    for i in count:
        threads[i].join()
    session.close()
    print("*** OVER ***")
